from time import sleep
import pandas as pd
import requests
from lxml import etree

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0'
}


def main():
    info_list = []
    for url in join_url():
        info_list += parse_html(req_url(url))
        sleep(2)
    save_info(info_list)


def join_url():
    url_list = [f'http://bang.dangdang.com/books/newhotsales/01.00.00.00.00.00-month-2023-{i}-1-{j}' for i in
                range(1, 12) for j in range(1, 26)]
    return url_list


def req_url(url):
    response = requests.get(url, headers=headers)
    return response.text


def parse_html(html):
    html = etree.HTML(html)
    li = html.xpath(
        '/html/body/div[@class="bang_wrapper"]/div[@class="bang_content"]/div[@class="bang_list_box"]/ul/li')
    info_list = []
    for i in li:
        rank = ''.join(i.xpath('./div[1]/text()'))
        name = ''.join(i.xpath('./div[@class="name"]/a/@title'))
        comment_count = ''.join(i.xpath('./div[@class="star"]/a/text()'))
        recommend_percent = ''.join(i.xpath('./div[@class="star"]/span[@class="tuijian"]/text()'))
        publisher_info_div = i.xpath('./div[@class="publisher_info"]')
        publisher_info = ''.join(
            [info for div in publisher_info_div for info in div.xpath('./a[1]/@title|./a[1]/text()|./span/text()')])
        price_n = ''.join(i.xpath('./div[@class="price"]/p/span[@class="price_n"]/text()'))
        price_r = ''.join(i.xpath('./div[@class="price"]/p/span[@class="price_r"]/text()'))
        price_s = ''.join(i.xpath('./div[@class="price"]/p/span[@class="price_s"]/text()'))
        info_list.append([rank, name, comment_count, recommend_percent, publisher_info, price_n, price_r, price_s])
    print(info_list)
    return info_list


def save_info(info_list):
    df = pd.DataFrame(info_list,
                      columns=['rank', 'name', 'comment_count', 'recommend_percent', 'publisher_info', 'price_n',
                               'price_r', 'price_s'])
    df.to_csv('../static/data/dangdang.csv', index=False, encoding='utf-8-sig')


if __name__ == '__main__':
    main()
